# coding=utf-8
'''
Created on 2011-4-18

@author: thinkpad
'''
import os
import codecs
import re
import json

from BeautifulSoup import *

class SinaDataParser:
    def __init__(self, base_dir):
        self.base_dir = base_dir

    def readFile(self, name):
        file = open(os.path.join(self.base_dir, name), 'r')
        html = file.read()
        file.close()
        return html

    def writeFile(self, name, str):
        file = open(os.path.join(self.base_dir, name), 'w')
        file.write(str)

    def city_to_json(self):
        html = self.readFile('city.txt')
        soup = BeautifulStoneSoup(html)
        dividor = soup.find('div', 'clearfix')

        #before is hot city
        hot_city_labels = dividor.findAllPrevious('label')
        hot_cities = []
        for city_label in reversed(hot_city_labels):
            city = self.parseCity(city_label)
            city['leaf'] = True
            city['checked'] = False
            hot_cities.append(city)
        #printJson(hot_cities)

        #after is cities for each province
        province_city_labels = dividor.findAllNext('label')
        provinces = {}
        other_id = 999999
        for city_label in province_city_labels:
            city = self.parseCity(city_label)
            city_id = int(city['id'])
            if (city_id % 10000 == 0 and city_id != 990000) or city_id == other_id:
                provinces[city_id] = city
            else:
                province_id = city_id - city_id % 10000
                if provinces.has_key(province_id):
                    province = provinces[province_id]
                else:
                    province = provinces[other_id]
                if not province.has_key('children'):
                    province['children'] = []
                city['leaf'] = True
                city['checked'] = False
                province['children'].append(city)
        sorted_keys = sorted(provinces.keys())
        sorted_provinces = map(lambda key: provinces[key], sorted_keys)

        cities = [{'text': u"热门城市", 'expanded': True, 'children': hot_cities},
                  {'text': u'其它城市', 'children': sorted_provinces}]
        self.saveJson(cities, 'city.json')

    def printJson(self, obj):
        print(json.dumps(obj, ensure_ascii=False, indent=4))

    def saveJson(self, obj, file_name):
        file_name = os.path.join(self.base_dir, file_name)
        with codecs.open(file_name, mode='w', encoding='utf-8') as fp:
            json.dump(obj, fp, ensure_ascii=False, indent=4)

    def parseCity(self, label):
        forAttr = label['for']
        id = self.__findId(forAttr)
        text = label.string
        return {'id': id, 'text': text}

    def __findId(self, str):
        p = re.compile('\D*(\d*)\D*')
        m = p.match(str)
        return m.group(1)

    def position_to_json(self):
        html = self.readFile('position.txt')
        soup = BeautifulStoneSoup(html)
        uls = soup.findAll('ul')
        parents = {}
        second = {}
        for ul in uls:
            hasId = False
            for attr in ul.attrs:
                if attr[0]=='id':
                    hasId = True
            if not hasId:
                #first level
                for li in ul.findAll('li'):
                    id = li['rel']
                    text = li.text
                    parents[id] = {'id': id, 'text': text}
            else:
                parentId = self.__findId(ul['id'])
                for li in ul.findAll('li'):
                    id = li.input['value']
                    text = li.text
                    sub_position = {'id': id, 'text': text, 'checked': False, 'leaf': True}
                    for parent in parents.values():
                        targetParent = self.__findParent(parent, parentId)
                        if targetParent is not None:
                            targetParent['children'].append(sub_position)
                            targetParent['leaf'] = False
                            break
#        self.printJson(parents)
        self.saveJson(parents.values(), 'position.json')

    def __findParent(self, parent, parentId):
        target = None
        if parent['id'] == parentId:
            target = parent
        elif parent.has_key('children'):
            for newParent in parent['children']:
                target = self.__findParent(newParent, parentId)
                if target is not None:
                    break
        if target is not None and not target.has_key('children'):
            target['children'] = []
        return target

    def industry_to_json(self):
        html = self.readFile('industry.txt')
        soup = BeautifulStoneSoup(html)
        uls = soup.findAll('ul')
        hot_ul, other_ul = uls
        industry_tuple = [(hot_ul, []), (other_ul, [])]
        for ul, industries in industry_tuple:
            for li in ul.findAll('li'):
                industries.append({'id': li.input['value'], 'text': li.text, 'checked': False, 'leaf': True})
        industries = [{'text': u"热门行业", 'expanded': True, 'children': industry_tuple[0][1]},
                  {'text': u'其它行业', 'children': industry_tuple[1][1]}]
#        self.printJson(industries)
        self.saveJson(industries, 'industry.json')

    def parse(self):
        self.city_to_json()
        self.position_to_json()
        self.industry_to_json()


if __name__ == '__main__':
    p = SinaDataParser('e:/code/javascript/sina/db/data')
    p.parse()
